package com.surfilter.phantomjs.util;

import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.surfilter.phantomjs.dto.WebInfo;
import net.sf.json.JSONObject;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.ParseException;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.jsoup.Connection;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.*;
import org.jsoup.select.Elements;
import org.openqa.selenium.*;
import org.openqa.selenium.phantomjs.PhantomJSDriver;
import org.openqa.selenium.phantomjs.PhantomJSDriverService;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.X509TrustManager;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.security.MessageDigest;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;

/**
 * 实现HTTP POST
 */
public final class HttpUtil {

    private static final Logger logger = LoggerFactory.getLogger(HttpUtil.class);

    private static final String phantomjsPath = "E:\\software\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe";

    private static final String crawlerSyncUrl = "https://dy.surfilter.com/?tag=common_page&m=common_page&m1=api&op=crawler_task";

    private static X509TrustManager tm = new X509TrustManager() {
        @Override
        public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
        }

        @Override
        public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
        }

        @Override
        public X509Certificate[] getAcceptedIssuers() {
            return null;
        }
    };

    private static SSLContext sslContext = null;

    private static Set<String> topd = new HashSet<String>();

    public static Set<String> titleBlacklist = new HashSet<String>();

    private static Set<String> excludeTags = new HashSet<String>();

    static {
        try {
            sslContext = SSLContext.getInstance("SSL", "SunJSSE");
            sslContext.init(null, new X509TrustManager[] { tm }, new java.security.SecureRandom());
        } catch (Exception e) {
            e.printStackTrace();
        }

        //网页标题黑名单
        titleBlacklist.add("阿里云-备案-阻断页-PC");
        titleBlacklist.add("TestPage");
        titleBlacklist.add("Apache Tomcat");
        titleBlacklist.add("阿里云万网虚机过期页面");
        titleBlacklist.add("访问出错");
        titleBlacklist.add("在从服务器接收结果时发生传输级错误");
        titleBlacklist.add("页面不存在_百度搜索");
        titleBlacklist.add("Default Web Site Page");
        titleBlacklist.add("bbs.esudai.com - Database Error");
        titleBlacklist.add("Forbidden 禁止访问");
        titleBlacklist.add("阿里云万网虚机手工关停页面");
        titleBlacklist.add("IIS7");


        // 双后缀域名先过滤
        topd.add(".hb.cn");
        topd.add(".net.cn");
        topd.add(".com.cn");
        topd.add(".yn.cn");
        topd.add(".ha.cn");
        topd.add(".he.cn");
        topd.add(".com.au");
        topd.add(".nx.cn");
        topd.add(".sd.cn");
        topd.add(".sn.cn");
        topd.add(".ah.cn");
        topd.add(".bj.cn");
        topd.add(".gx.cn");
        topd.add(".hl.cn");
        topd.add(".gd.cn");
        topd.add(".ac.cn");
        topd.add(".gs.cn");
        topd.add(".xj.cn");
        topd.add(".sc.cn");
        topd.add(".ln.cn");
        topd.add(".hk.cn");
        topd.add(".zj.cn");
        topd.add(".gov.cn");
        topd.add(".edu.cn");
        topd.add(".cn.com");
        topd.add(".hn.cn");
        topd.add(".gz.cn");
        topd.add(".js.cn");
        topd.add(".com.hk");
        topd.add(".sh.cn");
        topd.add(".fj.cn");
        topd.add(".nm.cn");
        topd.add(".net.au");
        topd.add(".mo.cn");
        topd.add(".tj.cn");
        topd.add(".jx.cn");
        topd.add(".jl.cn");
        topd.add(".qh.cn");
        topd.add(".cq.cn");
        topd.add(".org.cn");
        topd.add(".xz.cn");
        topd.add(".hi.cn");
        topd.add(".tw.cn");
        topd.add(".com.tw");
        topd.add(".sx.cn");

        excludeTags.add("head");
        excludeTags.add("script");
        excludeTags.add("style");
    }

    // 通用的http传输方法
    public static String post(String url, JSONObject json) throws IOException {
        URL url1 = null;
        BufferedReader reader = null;
        PrintWriter writer = null;
        OutputStreamWriter out = null;
        HttpURLConnection connection = null;
        try {
            url1 = new URL(url);

            connection = (HttpURLConnection) url1.openConnection();
            connection.setConnectTimeout(30000);
            connection.setReadTimeout(30000);
            connection.setRequestMethod("POST");
            connection.setInstanceFollowRedirects(true);

            connection.setDoInput(true);
            connection.setDoOutput(true);
            connection.setRequestProperty("Content-Type", "application/json;charset=UTF-8");
            connection.connect();
            out = new OutputStreamWriter(new BufferedOutputStream(connection.getOutputStream()), "UTF-8");
            out.write(json.toString());
            out.flush();
            out.close();
            System.out.println(connection.getResponseCode());
            if (connection.getResponseCode() >= 400) {
                throw new IllegalStateException("对方返回错误码:" + connection.getResponseCode());
            }
            reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"));
            StringBuffer sb = new StringBuffer();
            String line;
            while ((line = reader.readLine()) != null) {
                sb.append(line);
            }
            return sb.toString();
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e1) {
                }
            }
            if (writer != null) {
                writer.close();
            }
            if (out != null) {
                out.close();
            }
            if (connection != null)
                connection.disconnect();
        }
    }

    // 通用的http传输方法
    public static String post(String url, String params, Map<String, String> headers) throws IOException {
        URL url1 = null;
        BufferedReader reader = null;
        PrintWriter writer = null;
        OutputStreamWriter out = null;
        HttpURLConnection connection = null;
        try {
            url1 = new URL(url);

            connection = (HttpURLConnection) url1.openConnection();
            connection.setConnectTimeout(30000);
            connection.setReadTimeout(30000);
            connection.setRequestMethod("POST");
            connection.setInstanceFollowRedirects(true);

            connection.setDoInput(true);
            connection.setDoOutput(true);
            connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8");
            if(headers != null){
                for(String key : headers.keySet()){
                    connection.setRequestProperty(key, headers.get(key));
                }
            }
            connection.connect();
            out = new OutputStreamWriter(new BufferedOutputStream(connection.getOutputStream()), "UTF-8");
            out.write(params);
            out.flush();
            out.close();
            System.out.println(connection.getResponseCode());
//			if (connection.getResponseCode() >= 400) {
//				throw new IllegalStateException("对方返回错误码:" + connection.getResponseCode());
//			}
            reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"));
            StringBuffer sb = new StringBuffer();
            String line;
            while ((line = reader.readLine()) != null) {
                sb.append(line);
            }
            return sb.toString();
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e1) {
                }
            }
            if (writer != null) {
                writer.close();
            }
            if (out != null) {
                out.close();
            }
            if (connection != null)
                connection.disconnect();
        }
    }

    public static byte[] getPostByte(String url, JSONObject json) throws IOException {
        URL url1 = null;
        BufferedInputStream bufferedInputStream = null;
        OutputStreamWriter out = null;
        HttpURLConnection connection = null;

        url1 = new URL(url);

        connection = (HttpURLConnection) url1.openConnection();
        connection.setConnectTimeout(10000);
        connection.setReadTimeout(10000);
        connection.setRequestMethod("POST");
        connection.setInstanceFollowRedirects(true);

        connection.setDoInput(true);
        connection.setDoOutput(true);
        connection.setRequestProperty("Content-Type", "application/json;charset=UTF-8");
        connection.connect();
        out = new OutputStreamWriter(new BufferedOutputStream(connection.getOutputStream()), "UTF-8");
        out.write(json.toString());
        out.flush();
        out.close();
        System.out.println(connection.getResponseCode());
        if (connection.getResponseCode() >= 400) {
            throw new IllegalStateException("对方返回错误码:" + connection.getResponseCode());
        }

        bufferedInputStream = new BufferedInputStream(connection.getInputStream());
        ByteArrayOutputStream bos = new ByteArrayOutputStream(bufferedInputStream.available());

        try {

            int buf_size = 1024;
            byte[] buffer = new byte[buf_size];
            int len = 0;
            while (-1 != (len = bufferedInputStream.read(buffer, 0, buf_size))) {
                bos.write(buffer, 0, len);
            }
            bos.flush();
            bos.toByteArray();

            return bos.toByteArray();
        } finally {
            if (bufferedInputStream != null) {
                try {
                    bufferedInputStream.close();
                } catch (IOException e1) {
                }
            }
            if (connection != null) {
                connection.disconnect();
            }
        }
    }

    // 通用的http传输方法
    public static String get(String url, JSONObject json) throws IOException {
        URL url1 = null;
        BufferedReader reader = null;
        PrintWriter writer = null;
        OutputStreamWriter out = null;
        HttpURLConnection connection = null;
        try {
            url1 = new URL(url);

            connection = (HttpURLConnection) url1.openConnection();
            connection.setConnectTimeout(10000);
            connection.setReadTimeout(10000);
            connection.setRequestMethod("GET");
            connection.setInstanceFollowRedirects(true);

            connection.setDoInput(true);
            connection.setDoOutput(true);
            connection.setRequestProperty("Content-Type", "application/json;charset=UTF-8");
            connection.connect();
            out = new OutputStreamWriter(new BufferedOutputStream(connection.getOutputStream()), "UTF-8");
            if (json != null)
                out.write(json.toString());
            out.flush();
            out.close();
            System.out.println(connection.getResponseCode());
            if (connection.getResponseCode() >= 400) {
                throw new IllegalStateException("对方返回错误码:" + connection.getResponseCode());
            }
            String encoding = null;
            encoding = connection.getContentEncoding();

            reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), encoding));
            StringBuffer sb = new StringBuffer();
            String line;
            while ((line = reader.readLine()) != null) {
                sb.append(line);
            }
            return sb.toString();
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e1) {
                }
            }
            if (writer != null) {
                writer.close();
            }
            if (out != null) {
                out.close();
            }
            if (connection != null)
                connection.disconnect();
        }
    }

    // 通用的http传输方法
    public static String get(String url, Map<String, String> headers) throws IOException {
        URL url1 = null;
        BufferedReader reader = null;
        PrintWriter writer = null;
        OutputStreamWriter out = null;
        HttpURLConnection connection = null;
        try {
            url1 = new URL(url);

            connection = (HttpURLConnection) url1.openConnection();
            connection.setConnectTimeout(10000);
            connection.setReadTimeout(10000);
            connection.setRequestMethod("GET");
            connection.setDoOutput(true);
            if(headers != null){
                for(String key : headers.keySet()){
                    connection.setRequestProperty(key, headers.get(key));
                }
            }
            connection.connect();
            out = new OutputStreamWriter(new BufferedOutputStream(connection.getOutputStream()), "UTF-8");
            out.flush();
            out.close();
            System.out.println(connection.getResponseCode());
            if (connection.getResponseCode() >= 400) {
                throw new IllegalStateException("对方返回错误码:" + connection.getResponseCode());
            }
            String encoding = null;
            encoding = connection.getContentEncoding();

            reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), encoding));
            StringBuffer sb = new StringBuffer();
            String line;
            while ((line = reader.readLine()) != null) {
                sb.append(line);
            }
            return sb.toString();
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e1) {
                }
            }
            if (writer != null) {
                writer.close();
            }
            if (out != null) {
                out.close();
            }
            if (connection != null)
                connection.disconnect();
        }
    }

    public static String getHtmlTitle(String html){
        String result = null;
        try{
            if(html == null)return result;
            Pattern patt = Pattern.compile("<title[^<^>]*>([^<^>]*)</title>");
            Matcher match = patt.matcher(html);
            String retitle = null;
            while(match.find()){
                retitle = match.group();
                if(retitle.length() > 0)break;
            }
            if(retitle == null)return result;
            retitle = retitle.trim();
            result = retitle.replaceAll("</?title[^<^>]*>", "");
        }catch(Exception e){
            logger.error("获取HTML Title异常", e);
        }
        return result;
    }

    public static String getHtmlTitleByJsoup(String html){
        String result = "";
        try{
            result = Jsoup.parse(html).title();
        }catch(Exception e){
            logger.error("获取HTML Title异常", e);
        }
        return result;
    }

    public static String clearHtmlNotes(String html){
        String regex = "<!--[\\w\\W]*?-->";
        html = html.replaceAll(regex, "");
        return html;
    }

    public static String getHtmlDescription(String html){
        String result = null;
        try{
            String regex1 = "<meta\\s*(http-equiv|name)\\s*=\\s*\"(\\s*[kK][eE][yY][wW][oO][rR][dD][sS]|description|[kK][eE][yY][wW][oO][rR][dD])\\s*\"\\s*content\\s*=\\s*\"[^<^>]*\"\\s*>(([^<^>]*)</meta>)?";
            String regex2 = "content\\s*=\\s*\"[^<^>]*\"";
            String regex3 = "\".*\"";
            result = StringParseUtils.parseString(html, regex1, true);
            result = StringParseUtils.parseString(result, regex2, true);
            result = StringParseUtils.parseString(result, regex3, true);
            result = result.replaceAll("\"", "");
        }catch(Exception e){
            e.printStackTrace();
            logger.error("匹配Html Description异常", e);
        }
        return result;
    }

    public static String getHtmlBodyText(String html){
        String result = "";
        try{
            String regex1 = "<\\s*body\\s*[^<^>]*>([\\s\\S]*)</\\s*body\\s*[^<^>]*>";
            String regex2 = "<script[^<^>]*/?>[^<^>]*(</script[^<^>]*>)?";
            String regex3 = "<[^<^>]*>(</[^<^>]*>)?";
            result = StringParseUtils.parseString(html, regex1, true);
            result = result.replaceAll(regex2, "");
            result = result.replaceAll(regex3, "");
        }catch(Exception e){
            e.printStackTrace();
            logger.error("匹配 Body Text异常", e);
        }
        return result;
    }

    public static List<String> getHtmlBodyTexts(String html){
        List<String> result = new ArrayList<String>();
        try{
            String resultStr = "";
            String regex2 = "<script[^<^>]*/?>[^<^>]*(</script[^<^>]*>)?";
            String regex3 = "<[^<^>]*>(</[^<^>]*>)?";
            String regex4 = "<[^<^>]*>[^<^>]*</[^<^>]*>";
            resultStr = getHtmlBody(html);
            resultStr = resultStr.replaceAll(regex2, "");
            List<String> resultB = StringParseUtils.parseStringL(resultStr, regex4, true);
            if(resultB != null){
                for(String str : resultB){
                    result.add(str.replaceAll(regex3, ""));
                }
            }
        }catch(Exception e){
            e.printStackTrace();
            logger.error("匹配Body Texts异常", e);
        }
        return result;
    }

    public static List<String> getHtmlBodyTextsTwo(String html){
        List<String> result = new ArrayList<String>();
        try{
            String resultStr = "";
            String regex2 = "<script[^<^>]*/?>[^<^>]*(</script[^<^>]*>)?";
            String regex3 = "</?[^<^>]*>";
            String regex4 = "([\u4e00-\u9fa5]|[a-z]|[A-Z]|[0-9])+";
            resultStr = getHtmlBody(html);
            resultStr = resultStr.replaceAll(regex2, "");
            resultStr = resultStr.replaceAll(regex3, "");
            List<String> resultB = StringParseUtils.parseStringL(resultStr, regex4, true);
            result.addAll(resultB);
        }catch(Exception e){
            e.printStackTrace();
            logger.error("匹配Body Texts Two异常", e);
        }
        return result;
    }

    public static List<String> getHtmlBodyTextsThree(String html){
        List<String> result = new ArrayList<String>();
        try{
            String bodytext = "";
            bodytext = getHtmlBodyByJsoup(html);
            result.addAll(Arrays.asList(bodytext.split(" ")));
        }catch(Exception e){
            e.printStackTrace();
            logger.error("匹配Body Texts Three异常", e);
        }
        return result;
    }

    public static void cleanUpHidinTags(Node ele) {

        String disNone = null;
        if (ele == null)
            return;

        if(ele.hasAttr("style")){
            disNone = StringParseUtils.parseString(ele.attr("style"), "display\\s*:\\s*none", false);
        }

        if (ele.hasAttr("hidden") || (disNone != null && !disNone.trim().equalsIgnoreCase(""))
                || (ele.hasAttr("type") && ele.attr("type").contains("hidden"))) {
            ele.remove();
            return;
        }else{
            if(ele instanceof Comment){
                ele.remove();
            }else if(ele instanceof Element){
                Element subele = (Element)ele;
                String tagName = subele.tagName();
                if(excludeTags.contains(tagName)){
                    subele.remove();
                    return;
                }
                if(subele.children() == null || subele.children().size() == 0)
                    return;
                Elements eles = subele.children();
                for(int i = 0 ; i < eles.size() ; i ++){
                    cleanUpHidinTags(eles.get(i));
                }
            }else if(ele instanceof TextNode){
                System.out.println("TextNode :" + ele.nodeName());
            }else if(ele instanceof DataNode){
                System.out.println("DataNode :" + ele.nodeName());
            }else if(ele instanceof DocumentType){
                System.out.println("DocumentType :" + ele.nodeName());
            }else if(ele instanceof XmlDeclaration){
                System.out.println("XmlDeclaration :" + ele.nodeName());
            }
        }
    }

    public static String clearScript(String html){
        String result = html;
        if(!StringUtils.isEmpty(html) && !StringUtils.isBlank(html.trim())){
            String regex2 = "<script[^<^>]*/?>[^<^>]*(</script[^<^>]*>)?";
            result = result.replaceAll(regex2, "");
        }
        return result;
    }

    public static String getHtmlBody(String html){
        String body = null;
        String regex1 = "<\\s*body\\s*[^<^>]*>([\\s\\S]*)</\\s*body\\s*[^<^>]*>";
        if(!StringUtils.isEmpty(html) && !StringUtils.isBlank(html.trim())){
            body = StringParseUtils.parseString(html, regex1, true);
        }
        return body;
    }

    public static String getHtmlBodyByJsoup(String html){
        String body = null;
        Document doc = Jsoup.parse(html);
        body = doc.body().text();
        return body;
    }

    public static int getHtmlLabelNum(String html){
        int result = -1;
        try{
            result = StringParseUtils.parseLength(html, "<[^<^>]*>(</[^<^>]*>)?");
        }catch(Exception e){
            logger.error("获取Html标签个数异常,", e);
        }
        return result;
    }

    public static int visitException(List<String> htmls){
        int result = 3;
        if(htmls != null && !htmls.isEmpty()){
            int htmlLabelNum = 0;
            int htmlLength = 0;
            for(String html : htmls){
                String title = getHtmlTitle(html);
                if(title != null && !title.trim().equalsIgnoreCase("") &&isExsit(title)){
                    return 1;
                }
                html = getHtmlBody(html);
                html = clearScript(html);
                htmlLabelNum += getHtmlLabelNum(html);
                htmlLength += html.length();
            }
            if(htmlLabelNum < 50 && htmlLength < 5000)result = 1;
            else result = 2;
        }
        return result;
    }

    public static boolean isExsit(String str){
        boolean target = false;
        if(titleBlacklist.contains(str))return true;
        for(String titleBlack : titleBlacklist){
            if(titleBlack.contains(str)){
                target = true;
                break;
            }
        }
        return target;
    }


    public static String get(String url, String ecoding) throws IOException {
        URL url1 = null;
        BufferedReader reader = null;
        PrintWriter writer = null;
        OutputStreamWriter out = null;
        HttpURLConnection connection = null;
        try {
            url1 = new URL(url);

            connection = (HttpURLConnection) url1.openConnection();
            if (url.startsWith("https")) {
                HttpsURLConnection hscon = (HttpsURLConnection) connection;
                hscon.setSSLSocketFactory(sslContext.getSocketFactory());
            }
            connection.setConnectTimeout(10000);
            connection.setReadTimeout(10000);
            connection.setRequestMethod("GET");
            connection.setInstanceFollowRedirects(true);
            connection.setDoInput(true);
            connection.setDoOutput(true);
//			tnet=111.205.62.10; TYCID=b432a0ffe4bf496a9b30e9bc3fed1b18; _pk_ref.1.e431=%5B%22%22%2C%22%22%2C1466659898%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DBd5vW0eWsdqWK12oURfg4f6gc15YJt33AszbXecJgAUWWZqPeOSh-42Jh9IpIb59%26wd%3D%26eqid%3Deaad6d450000eb2300000002576b5227%22%5D; _pk_id.1.e431=5044394bcf92ce92.1466656724.2.1466660117.1466659898.; _pk_ses.1.e431=*; Hm_lvt_e92c8d65d92d534b0fc290df538b4758=1466659898,1466660117; Hm_lpvt_e92c8d65d92d534b0fc290df538b4758=1466660117; token=239780cb749442879ed46eae05274a64; _utm=4b6327a3a8074953a0182b2205d3588f
//			connection.setRequestProperty("Content-Type", "application/json;charset=UTF-8");
            connection.setRequestProperty("Accept", "application/json, text/plain, */*");
            connection.setRequestProperty("Accept-Encoding", "gzip,deflate,sdch");
            connection.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.8");
            connection.setRequestProperty("CheckError", "check");
            connection.setRequestProperty("Connection", "keep-alive");
            connection.setRequestProperty("Pragma", "no-cache");
            connection.setRequestProperty("Upgrade-Insecure-Requests", "1");
            connection.setRequestProperty("User-Agent",
                    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36");
            connection.setRequestProperty("Cookie", "tnet=111.205.62.10; TYCID=b432a0ffe4bf496a9b30e9bc3fed1b18; _pk_ref.1.e431=%5B%22%22%2C%22%22%2C1466659898%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DBd5vW0eWsdqWK12oURfg4f6gc15YJt33AszbXecJgAUWWZqPeOSh-42Jh9IpIb59%26wd%3D%26eqid%3Deaad6d450000eb2300000002576b5227%22%5D; _pk_id.1.e431=5044394bcf92ce92.1466656724.2.1466660117.1466659898.; _pk_ses.1.e431=*; Hm_lvt_e92c8d65d92d534b0fc290df538b4758=1466659898,1466660117; Hm_lpvt_e92c8d65d92d534b0fc290df538b4758=1466660117; token=239780cb749442879ed46eae05274a64; _utm=4b6327a3a8074953a0182b2205d3588f");
            connection.connect();
            if (connection.getResponseCode() >= 400) {
                throw new IllegalStateException("对方返回错误码:" + connection.getResponseCode());
            }
            String encoding = null;
            encoding = connection.getContentEncoding();
            if(encoding == null)encoding = ecoding;
            reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), encoding));
            StringBuffer sb = new StringBuffer();
            String line;
            while ((line = reader.readLine()) != null) {
                sb.append(line);
            }
            return sb.toString();
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e1) {
                }
            }
            if (writer != null) {
                writer.close();
            }
            if (out != null) {
                out.close();
            }
            if (connection != null)
                connection.disconnect();
        }
    }

    public static void gettyc(String url, String ecoding, String file_path) throws IOException {
        URL url1 = null;
        InputStream reader = null;
        PrintWriter writer = null;
        OutputStream out = null;
        HttpURLConnection connection = null;
        try {
            url1 = new URL(url);

            connection = (HttpURLConnection) url1.openConnection();
            if (url.startsWith("https")) {
                HttpsURLConnection hscon = (HttpsURLConnection) connection;
                hscon.setSSLSocketFactory(sslContext.getSocketFactory());
            }
            connection.setConnectTimeout(10000);
            connection.setReadTimeout(10000);
            connection.setRequestMethod("GET");
            connection.setInstanceFollowRedirects(true);
            connection.setDoInput(true);
            connection.setDoOutput(true);
//			tnet=111.205.62.10; TYCID=b432a0ffe4bf496a9b30e9bc3fed1b18; _pk_ref.1.e431=%5B%22%22%2C%22%22%2C1466659898%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DBd5vW0eWsdqWK12oURfg4f6gc15YJt33AszbXecJgAUWWZqPeOSh-42Jh9IpIb59%26wd%3D%26eqid%3Deaad6d450000eb2300000002576b5227%22%5D; _pk_id.1.e431=5044394bcf92ce92.1466656724.2.1466660117.1466659898.; _pk_ses.1.e431=*; Hm_lvt_e92c8d65d92d534b0fc290df538b4758=1466659898,1466660117; Hm_lpvt_e92c8d65d92d534b0fc290df538b4758=1466660117; token=239780cb749442879ed46eae05274a64; _utm=4b6327a3a8074953a0182b2205d3588f
//			connection.setRequestProperty("Content-Type", "application/json;charset=UTF-8");
            connection.setRequestProperty("Accept", "application/json, text/plain, */*");
            connection.setRequestProperty("Accept-Encoding", "gzip,deflate,sdch");
            connection.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.8");
            connection.setRequestProperty("CheckError", "check");
            connection.setRequestProperty("Connection", "keep-alive");
            connection.setRequestProperty("Pragma", "no-cache");
            connection.setRequestProperty("Upgrade-Insecure-Requests", "1");
            connection.setRequestProperty("User-Agent",
                    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36");
            connection.setRequestProperty("Cookie", "tnet=111.205.62.10; TYCID=b432a0ffe4bf496a9b30e9bc3fed1b18; _pk_ref.1.e431=%5B%22%22%2C%22%22%2C1466659898%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DBd5vW0eWsdqWK12oURfg4f6gc15YJt33AszbXecJgAUWWZqPeOSh-42Jh9IpIb59%26wd%3D%26eqid%3Deaad6d450000eb2300000002576b5227%22%5D; _pk_id.1.e431=5044394bcf92ce92.1466656724.2.1466660117.1466659898.; _pk_ses.1.e431=*; Hm_lvt_e92c8d65d92d534b0fc290df538b4758=1466659898,1466660117; Hm_lpvt_e92c8d65d92d534b0fc290df538b4758=1466660117; token=239780cb749442879ed46eae05274a64; _utm=4b6327a3a8074953a0182b2205d3588f");
            connection.connect();
            if (connection.getResponseCode() >= 400) {
                throw new IllegalStateException("对方返回错误码:" + connection.getResponseCode());
            }
            String encoding = null;
            encoding = connection.getContentEncoding();
            if(encoding == null)encoding = ecoding;
            reader = connection.getInputStream();
            out = new FileOutputStream(new File(file_path));
            byte[] datas = new byte[10240];
            int num = 0;
            while ((num = reader.read(datas)) != -1) {
                out.write(datas, 0, num);
            }
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e1) {
                }
            }
            if (writer != null) {
                writer.close();
            }
            if (out != null) {
                out.close();
            }
            if (connection != null)
                connection.disconnect();
        }
    }

    public static String saveUrlToFile(String url, String filepath) throws Exception {
        Connection con = Jsoup.connect(url);
        con.validateTLSCertificates(false);

        con.timeout(30000);
        con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        con.header("Accept-Encoding", "gzip, deflate, sdch");
        con.header("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4");
        con.header("Cache-Control", "no-cache");
        con.header("Connection", "keep-alive");
        con.header("Pragma", "no-cache");
        con.header("Upgrade-Insecure-Requests", "1");
        con.header("User-Agent",
                "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36");

        Document doc = con.get();
        if (doc != null) {
            String html = doc.html();
            PrintWriter w = new PrintWriter(filepath);
            w.println(html);
            w.flush();
            w.close();
        }

        //System.out.println(filepath);
        return filepath;
    }

    /**
     * 通过domain(域名)获取网站Html标签,并将Html标签存入指定文件
     * 返回正确打开网站的Url
     */
    public static String saveWebHtmlByDomain(String domain, String file_path){
        String url = null;
        List<String> urls = getUrls(domain);
        for(String rurl : urls){
            Response resp = getResponse(rurl);
            if(resp == null)continue;
            try {
                Document doc = resp.parse();
                if(doc == null)continue;
                String html = doc.html();
                if(html == null || html.trim().equalsIgnoreCase(""))continue;

            } catch (Exception e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
                logger.error("打开网站：{}异常", rurl, e);
                continue;
            }

        }
        return url;
    }

    public static boolean webJump(Document doc){
        boolean target = false;

        String metaUrl = null;
        Elements head = doc.getElementsByTag("head");
        Elements meta = doc.select("meta[http-equiv=refresh]");
        if(meta != null && meta.size() > 0){
            metaUrl = meta.get(0).attr("content");
        }else{
            metaUrl = null;
        }

        if(metaUrl == null){
            if(head != null && head.size() > 0){
                String values = StringParseUtils.parseString(head.toString(), "\\s*meta\\s*http-equiv\\s*=\\s*\\'?refresh\\'?\\s*content\\s*=\\s*\\'.+\\'", false);
                if(values != null && !values.trim().equalsIgnoreCase("")){
                    metaUrl = values;
                }
            }
        }
        if(metaUrl == null){
            String html = doc.html();
            int labelNum = getHtmlLabelNum(html);
            if(labelNum < 100 || html.length() < 5000){
                Elements scriptEle = doc.getElementsByTag("script");
                Elements iframeEle = doc.getElementsByTag("iframe");
                Elements frameEle = doc.getElementsByTag("frame");
                Element bodyEle = doc.body();
                //判断iframe标签存在否
                if(iframeEle != null && iframeEle.size() > 0 && !target){
                    for(int i = 0; i < iframeEle.size(); i++){
                        String framesrc = iframeEle.get(i).attr("src");
                        if(framesrc != null && !framesrc.trim().equals("")){
                            target = true;
                            break;
                        }
                    }
                }
                //判断frame标签存在否
                if(frameEle != null && frameEle.size() > 0 && !target){
                    for(int i = 0; i < frameEle.size(); i++){
                        String framesrc = frameEle.get(i).attr("src");
                        if(framesrc != null && !framesrc.trim().equals("")){
                            target = true;
                            break;
                        }
                    }
                }

                //判断js跳转
                if(scriptEle != null && scriptEle.size() > 0  && !target){
                    String script_str = scriptEle.toString();
                    script_str = script_str.replaceAll("function\\s+[^\\(]+\\([^\\(]*\\)\\s*\\{[\\s\\S]*\\}", "");
                    String values = StringParseUtils.parseString(script_str, ".*\\.?location(\\.[a-z]*)?\\s*=.*", false);
                    if((values != null && !values.trim().equalsIgnoreCase("")) || script_str.contains("window.onload")){
                        target = true;
                    }else{
                        target = (script_str.contains("XMLHttpRequest") || script_str.contains(".submit"));
                    }
                }
                //判断body跳转存在不
                if(bodyEle != null && !target){
                    String bodyOnload = bodyEle.attr("onload");
                    if(bodyOnload != null && !bodyOnload.trim().equalsIgnoreCase(""))target = true;
                }
            }
        }else{
            target = true;
        }
        return target;
    }

    public static String saveUrlToFileRHtml(String url, String filepath) throws Exception {
        String html = null;
        Connection con = Jsoup.connect(url);
        con.validateTLSCertificates(false);

        con.timeout(30000);
        con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        con.header("Accept-Encoding", "gzip, deflate, sdch");
        con.header("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4");
        con.header("Cache-Control", "no-cache");
        con.header("Connection", "keep-alive");
        con.header("Pragma", "no-cache");
        con.header("Upgrade-Insecure-Requests", "1");
        con.header("User-Agent",
                "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36");

        Document doc = con.get();
        if (doc != null) {
            html = doc.html();
            PrintWriter w = new PrintWriter(filepath);
            w.println(html);
            w.flush();
            w.close();
        }

        //System.out.println(filepath);
        return html;
    }

    public static Document saveUrlToFileRDoc(String url, String filepath) throws Exception {
        Connection con = Jsoup.connect(url);
        con.validateTLSCertificates(false);

        con.timeout(30000);
        con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        con.header("Accept-Encoding", "gzip, deflate, sdch");
        con.header("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4");
        con.header("Cache-Control", "no-cache");
        con.header("Connection", "keep-alive");
        con.header("Pragma", "no-cache");
        con.header("Upgrade-Insecure-Requests", "1");
        con.header("User-Agent",
                "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36");

        Document doc = con.get();
        if (doc != null) {
            String html = doc.html();
            PrintWriter w = new PrintWriter(filepath);
            w.println(html);
            w.flush();
            w.close();
        }

        //System.out.println(filepath);
        return doc;
    }


    /**
     *  如果url不含有http则给其拼接上http
     *	如果url不含有https则给其拼接上https
     *	如果url不含有www.则给其拼接上www.
     *	返回真实可访问的URL,否则返回空
     */
    public static String saveSnapshot(String ym, String filepath){
        String result = null;
        List<String> results = getUrls(ym);
        for(String ur : results){
            try {
                saveUrlToFile(ur, filepath);
                result = ur;
                break;
            } catch (Exception e) {
                //	System.out.println(2);
                logger.error("获取网站快照异常:{}", e);
            }
        }
        return result;
    }

    public static boolean saveSnapshotb(String url, String filepath){
        boolean  result = false;
        List<String> results = getUrls(url);
        for(String ur : results){
            try {
                saveUrlToFile(ur, filepath);
                result = true;
                break;
            } catch (Exception e) {
                // TODO Auto-generated catch block
                logger.error("获取网站快照异常:{}", e);
            }
        }
        return result;
    }

    public static String saveSnapshotb_url(String url, String filepath){
        String  result = null;
        List<String> results = getUrls(url);
        for(String ur : results){
            try {
                String fp = saveUrlToFile(ur, filepath);
                if(fp!=null){
                    result = ur;
                }
                break;
            } catch (Exception e) {
                e.printStackTrace();
                logger.error("获取快照异常", e);
            }
        }
        return result;
    }

    public static Document saveSnapshotb_url_doc(String url, String filepath){
        Document  result = null;
        List<String> results = getUrls(url);
        for(String ur : results){
            try {
                Document fp = saveUrlToFileRDoc(ur, filepath);
                if(fp!=null){
                    result = fp;
                }
                break;
            } catch (Exception e) {
                // TODO Auto-generated catch block
                //System.out.println(result);
                //e.printStackTrace();
            }
        }
        return result;
    }

    public static List<String> getUrls(String url){
        List<String> lists = new ArrayList<String>();
        if(url == null || url.trim().equalsIgnoreCase(""))return lists;
        url = url.trim();
        if(!url.contains("http")){
            if(!url.contains("www.")){
                lists.add("http://www." + url);
                lists.add("http://" + url);
                lists.add("https://www." + url);
                lists.add("https://" + url);
            }else{
                lists.add("http://" + url);
                lists.add("https://" + url);
            }
        }else{
            lists.add(url);
        }
        return lists;
    }

    @Deprecated
    public static String saveUrlToEncodeFile(String url, String filepath) throws Exception {
        URL url1 = null;
        HttpURLConnection sconnection = null;
        InputStream is = null;
        OutputStream os = null;
        try {
            url1 = new URL(url);
            sconnection = (HttpURLConnection) url1.openConnection();
            if (url.startsWith("https")) {
                HttpsURLConnection hscon = (HttpsURLConnection) sconnection;
                hscon.setSSLSocketFactory(sslContext.getSocketFactory());
            }
            sconnection.setConnectTimeout(10000);
            sconnection.setReadTimeout(10000);
            sconnection.setRequestMethod("GET");
            sconnection.setDoOutput(true);

            sconnection.setRequestProperty("Accept",
                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            sconnection.setRequestProperty("Accept-Encoding", "gzip, deflate, sdch");
            sconnection.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4");
            sconnection.setRequestProperty("Cache-Control", "no-cache");
            sconnection.setRequestProperty("Connection", "keep-alive");
            sconnection.setRequestProperty("Pragma", "no-cache");
            sconnection.setRequestProperty("Upgrade-Insecure-Requests", "1");
            sconnection.setRequestProperty("User-Agent",
                    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.108 Safari/537.36");
            sconnection.connect();
            if (sconnection.getResponseCode() >= 400) {
                throw new IllegalStateException(url + "\t对方返回错误码:" + sconnection.getResponseCode());
            }
            String contentType = sconnection.getContentType();
            String cencoding = sconnection.getHeaderField("Content-Encoding");
            String encode = null;
            int ccindex = contentType.indexOf("charset=");
            if (ccindex >= 0) {
                encode = contentType.substring(ccindex + 8);
                int inx = encode.indexOf(";");
                if (inx > 0) {
                    encode = encode.substring(0, inx).trim().toUpperCase();
                }
                if (encode.contains("ASCII"))
                    encode = "UTF-8";
                else if (encode.contains("GBK") || encode.contains("GB2312") || encode.contains("ISO-8859-2")
                        || encode.contains("EUC-TW"))
                    encode = "GB18030";
            } else {
                encode = "UNKNOW";
            }
            is = sconnection.getInputStream();
            InputStream gis = null;
            if (cencoding != null && cencoding.toLowerCase().contains("gzip")) {
                gis = new GZIPInputStream(is);
            } else {
                gis = is;
            }
            os = new FileOutputStream(filepath + "." + encode);
            int ch = -1;
            while ((ch = gis.read()) != -1) {
                os.write(ch);
            }
            return filepath + "." + encode;
        } catch (Exception e) {
            throw e;
        } finally {
            if (sconnection != null)
                sconnection.disconnect();
            if (is != null)
                is.close();
            if (os != null)
                os.close();
        }
    }
    /**
     * 获取内网快照的 title。。。。
     * @param nwUrl   内网快照地址
     * */
    public static Map<String,Object>  getNwInfo(String nwUrl){
        Map<String,Object> map = new HashMap<String,Object>();
        String titleOld= "";
        String keywordsOld= "";
        String descriptionOld= "";
        File file = new File(nwUrl);
        Document doc;
        try {
            doc = Jsoup.parse(file,"UTF-8","");
            //	String html = doc.html();
            if(doc != null){

                if(doc.title() != null && !doc.title().equals("")){
                    titleOld = doc.title();
                }else{
                    titleOld = null;
                }
                //keywordsOld =  doc.select("meta[name=keywords]").attr("content");
                if(doc.select("meta[name=keywords]") != null){
                    keywordsOld =  doc.select("meta[name=keywords]").attr("content");
                }else{
                    keywordsOld = null;
                }

                if(doc.select("meta[name=description]") != null){
                    descriptionOld =  doc.select("meta[name=description]").attr("content");
                }else{
                    descriptionOld = null;
                }

                map.put("keywordsOld", keywordsOld);
                map.put("titleOld", titleOld);
                map.put("descriptionOld", descriptionOld);
                return map;
            }
        } catch (Exception e) {
            // TODO Auto-generated catch block
            logger.error("获取快照信息异常", e);
        }

        return null;
    }
    /**
     * 获取外网快照的 title。。。。
     * @param url 外网地址
     * @throws Exception
     * */
    public static Map<String,Object>  getWwInfo(String url, String filePath) throws Exception{
        Map<String,Object> map = new HashMap<String,Object>();
        String titleNew= "";
        String keywordsNew= "";
        String descriptionNew= "";
        String html ="";

        Document doc = saveSnapshotb_url_doc(url, filePath);
        try {
            if(doc != null){
                html= doc.html();

                if(doc.title() != null && !doc.title().equals("")){
                    titleNew = doc.title();
                }else{
                    titleNew = null;
                }

                if(doc.select("meta[name=keywords]") != null){
                    keywordsNew = doc.select("meta[name=keywords]").attr("content");
                }else{
                    keywordsNew = null;
                }

                if(doc.select("meta[name=description]") != null){
                    descriptionNew = doc.select("meta[name=description]").attr("content");
                }else{
                    descriptionNew = null;
                }

                map.put("titleNew", titleNew);
                map.put("keywordsNew", keywordsNew);
                map.put("descriptionNew", descriptionNew);
                return map;
            }
        } catch (Exception e) {
            logger.error("获取外网快照信息异常", e);
        }

        return null;

    }

//	public static void main(String[] args) {
//		Map<String,Object> map = new HashMap<String,Object>();
//		String titleOld= "";
//		String keywordsOld= "";
//		String descriptionOld= "";
//		File file = new File("/datanfs/rzx/sync/snapshot/F5E29B986C6BDEF32CB8594C9694C600");
//		Document doc;
//		try {
//			doc = Jsoup.parse(file,"UTF-8","");
//			//String html = doc.html();
//			titleOld = doc.title();
//			keywordsOld =  doc.select("meta[name=keywords]").attr("content");
//			descriptionOld =  doc.select("meta[name=description]").attr("content");
//			System.out.println(titleOld);
//			System.out.println(keywordsOld);
//			System.out.println(descriptionOld);
//			map.put("keywordsOld", keywordsOld);
//			map.put("titleOld", titleOld);
//			map.put("descriptionOld", descriptionOld);
//		} catch (IOException e) {
//			// TODO Auto-generated catch block
//			e.printStackTrace();
//
//		}
//	}

    //获取元搜索关键字
    public static List<String> querykeywords(String path){
        List<String> keywords = null;
        try {
            keywords = org.apache.commons.io.FileUtils.readLines(new File(path),"UTF-8");
            System.out.println(keywords);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return keywords;
    }



//	public static void main(String[] args) throws Exception {
////		downandcal(args);
////		 testsaveurl(args);
//		// testcalword(args);
//		 testSnap("C:\\Users\\xubin\\workspace\\funds_task\\parsnum", 100);
//	}
//
//	private static int num;
//
//	public static synchronized void addNum(){
//		num ++;
//		System.out.println(num);
//	}
//
//	public static void testSnap(String file, int threadNum) throws IOException{
//		ExecutorService send = Executors.newFixedThreadPool(threadNum);
//		List<String> results = FileUtils.readLines(new File(file), "utf-8");
//		for(final String domain : results){
//			Thread thread = new Thread(){
//				@Override
//				public void run() {
//					// TODO Auto-generated method stub
//					if(!saveSnapshotb(domain, "d:/html")){
//						System.out.println(domain);
//					}
//					addNum();
//				}
//			};
//			send.execute(thread);
//		}
//	}


//	private static void downandcal(String[] args) throws Exception {
//
//		ExecutorService send = Executors.newFixedThreadPool(200);
//		if (args.length <= 1)
//			System.out.println("参数错误!");
//		String type = args[0];
//		if ("1".equals(type)) {// 下载url，输出到文件
//			final String urlfile = args[1];
//			final String savepath = args[2];
//			BufferedReader reader = new BufferedReader(new FileReader(urlfile));
//			try {
//				while (true) {
//					String fline = reader.readLine();
//					if (fline == null)
//						break;
//					if (fline.trim().length() == 0)
//						continue;
//					final String url = "http://" + fline.trim();
//					try {
//						send.execute(new Runnable() {
//							@Override
//							public void run() {
//								try {
//									saveUrlToFile(url.trim(), savepath + getDomainFromUrl(url));
//								} catch (Exception e) {
//									System.out.println(url);
//								}
//							}
//						});
//					} catch (Exception e) {
//						// e.printStackTrace();
//						continue;
//					}
//				}
//			} catch (Exception e) {
//				// e.printStackTrace();
//			} finally {
//				reader.close();
//			}
//			Thread.sleep(10000);
//			send.shutdown();
//			while (true) {
//				if (send.isTerminated()) {
//					System.out.println("下载url结束了!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
//					break;
//				}
//				Thread.sleep(200);
//			}
//		} else if ("2".equals(type)) {// 验证独立词，输出到文件
//			final HashMap<String, Integer> keywords = getJRKeywords();
//			final String urlfile = args[1];
//			final String savepath = args[2];
//			final String restfile = args[3];
//			BufferedReader reader = new BufferedReader(new FileReader(urlfile));
//			final PrintWriter writer = new PrintWriter(restfile);
//			try {
//				while (true) {
//					String fline = reader.readLine();
//					if (fline == null)
//						break;
//					fline = fline.trim();
//					String url = "http://" + fline.trim();
//					final String domain = getDomainFromUrl(url);
//					final String htmlfile = savepath + domain;
//					try {
//						send.execute(new Runnable() {
//							@Override
//							public void run() {
//								try {
//									int value = DocumentUtil.calcWordCapInHtmlFile(keywords, htmlfile);
//									if (value != -1)
//										writeFile(writer, domain + "\t" + value);
//								} catch (Exception e) {
//									e.printStackTrace();
//								}
//							}
//						});
//					} catch (
//
//					Exception e)
//
//					{
//						// e.printStackTrace();
//						continue;
//					}
//				}
//			} catch (Exception e) {
//				// e.printStackTrace();
//			} finally {
//				reader.close();
//			}
//			Thread.sleep(10000);
//			send.shutdown();
//			while (true)
//
//			{
//				if (send.isTerminated()) {
//					System.out.println("计算独立词结束了!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
//					writer.close();
//					break;
//				}
//				Thread.sleep(200);
//			}
//
//		} else if ("3".equals(type)) {// 提取排除词
//			final String urlfile = args[1];
//			final String savepath = args[2];
//			final String restfile = args[3];
//			BufferedReader reader = new BufferedReader(new FileReader(urlfile));
//			final PrintWriter writer = new PrintWriter(restfile);
//			try {
//				while (true) {
//					String fline = reader.readLine();
//					if (fline == null)
//						break;
//					fline = fline.trim();
//					String url = "http://" + fline.trim();
//					final String domain = getDomainFromUrl(url);
//					final String htmlfile = savepath + domain;
//					try {
//						send.execute(new Runnable() {
//							@Override
//							public void run() {
//								try {
//									HashMap<String, Integer> keymap = DocumentUtil
//											.getIsolatedWordFromHtmlFileExBody(htmlfile);
//									if (keymap == null)
//										return;
//									Set<String> keyset = keymap.keySet();
//									for (String key : keyset) {
//										if (key.length() > 3 && key.length() < 9)
//											writeFile(writer, key);
//									}
//
//								} catch (Exception e) {
//									e.printStackTrace();
//								}
//							}
//						});
//					} catch (Exception e) {
//						// e.printStackTrace();
//						continue;
//					}
//				}
//			} catch (Exception e) {
//				// e.printStackTrace();
//			} finally {
//				reader.close();
//			}
//			Thread.sleep(10000);
//			send.shutdown();
//			while (true) {
//				if (send.isTerminated()) {
//					System.out.println("提取排除词结束了");
//					writer.close();
//					break;
//				}
//				Thread.sleep(200);
//			}
//
//		} else if ("4".equals(type)) {// 排除筛选
//			final String urlfile = args[1];
//			final String savepath = args[2];
//			final String restfile = args[3];
//			BufferedReader reader = new BufferedReader(new FileReader(urlfile));
//			final PrintWriter writer = new PrintWriter(restfile);
//			final String[] keywords = getNotJRKeywords();
//			try {
//				while (true) {
//					String fline = reader.readLine();
//					if (fline == null)
//						break;
//					fline = fline.trim();
//					String url = "http://" + fline.trim();
//					final String domain = getDomainFromUrl(url);
//					final String htmlfile = savepath + domain;
//					try {
//						send.execute(new Runnable() {
//							@Override
//							public void run() {
//								try {
//									int value = DocumentUtil.isIncludeWords(keywords, htmlfile);
//									writeFile(writer, domain + "\t" + value);
//								} catch (Exception e) {
//									e.printStackTrace();
//								}
//							}
//						});
//					} catch (Exception e) {
//						// e.printStackTrace();
//						continue;
//					}
//				}
//			} catch (Exception e) {
//				// e.printStackTrace();
//			} finally {
//				reader.close();
//			}
//			Thread.sleep(10000);
//			send.shutdown();
//			while (true) {
//				if (send.isTerminated()) {
//					System.out.println("排除词结束了");
//					writer.close();
//					break;
//				}
//				Thread.sleep(200);
//			}
//
//		}
//	}

    public static String ip138(String ip){
        String target = null;
        try{
            String url = "http://www.ip138.com/ips138.asp?ip=" + ip + "&action=2";
            Document doc = HttpUtil.getWebDocument(url);
            if (doc == null) return target;
            Elements eles = doc.getElementsByClass("ul1");
            if(eles != null && eles.size() > 0){
                Element ele = eles.get(0);
                Elements childrens = ele.children();
                if(childrens != null && childrens.size() > 0){
                    for(int i = 0 ; i < childrens.size() ; i ++){
                        Element chele = childrens.get(i);
                        String text = chele.ownText();
                        if(text != null && text.contains("本站数据：")){
                            target = text.replaceAll("本站数据：", "");
                        }
                    }
                }
            }

        }catch(Exception e){
            e.printStackTrace();
            logger.error("通过IP获取地市信息异常", e);
        }
        logger.info("测试 ---- 通过ip138()爬取网页获取得值为：" + target);
        return target;
    }

    public static String phone138(String phone){
        String target = null;
        try{
            String url = "http://www.ip138.com:8080/search.asp?mobile=" + phone +"&action=mobile";
            Document doc = HttpUtil.getWebDocument(url);
            Elements eles = doc.getElementsByClass("tdc");
//			Elements eles = doc.getElementsMatchingText("卡号归属地");
            if(eles != null && eles.size() > 0){
                Element ele = eles.get(4);
                Elements childrens = ele.children();
                if(childrens != null && childrens.size() > 0){
                    for(int i = 0 ; i < childrens.size() ; i ++){
                        Element chele = childrens.get(i);
                        Elements elements = chele.getElementsByClass("tdc2");
                        if(elements != null &&  !elements.isEmpty()){
                            String ownText = elements.get(0).ownText();
                            return ownText;
                        }
                    }
                }
            }
        }catch(Exception e){
            e.printStackTrace();
            logger.error("通过电话号码获取地市信息异常", e);
        }
        return target;
    }

    public static Response getResponse(String url){
        if(!StringUtils.isEmpty(url)){
            try {
                Response response = Jsoup.connect(url).validateTLSCertificates(false)
                        .userAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21")
                        .timeout(30000).header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
                        .header("Accept-Encoding", "gzip, deflate, sdch").header("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4")
                        .header("Connection", "keep-alive").header("Cache-Control", "no-cache").header("Pragma", "no-cache")
                        .header("Upgrade-Insecure-Requests", "1")
                        .execute();
                return response;
            } catch (Exception e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
                logger.error("jsoup打开网站异常", e);
            }
            return null;
        }
        throw new RuntimeException("Url不能为空或者null。。");
    }

    public static Response getResponse(String url, int timeout){
        if(!StringUtils.isEmpty(url)){
            try {
                Response response = Jsoup.connect(url).validateTLSCertificates(false)
                        .userAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21")
                        .timeout(timeout).header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
                        .header("Accept-Encoding", "gzip, deflate, sdch").header("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4")
                        .header("Connection", "keep-alive").header("Cache-Control", "no-cache").header("Pragma", "no-cache")
                        .header("Upgrade-Insecure-Requests", "1")
                        .execute();
                return response;
            } catch (Exception e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
                logger.error("jsoup打开网站:{}异常, 异常信息:{}", url, e.getMessage());
            }
            return null;
        }
        throw new RuntimeException("Url不能为空或者null。。");
    }

    public static Document getCCWebDocument(String url){
        Response response = getResponse(url);
        if(response == null)return null;
        try {

            Document parse = response.parse();
//			URL url2 = response.url();   //重定向之后的url
            return parse;

        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
            logger.error("jsoup解析异常", e);

        }
        return null;
    }



    public static Document getWebDocument(String url) {
        Document doc = null;
        Connection con = null;
        try{
            con = Jsoup.connect(url);
            con.validateTLSCertificates(false);

            con.timeout(30000);
            con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            con.header("Accept-Encoding", "gzip, deflate, sdch");
            con.header("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4");
            con.header("Cache-Control", "no-cache");
            con.header("Connection", "keep-alive");
            con.header("Pragma", "no-cache");
            con.header("Upgrade-Insecure-Requests", "1");
            con.header("User-Agent",
                    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36");

            doc = con.get();
//			con.execute().parse()
//
//			Response response = Jsoup.connect(url).validateTLSCertificates(true)
//					.userAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21")
//					.timeout(10000)
//					.execute();
//			URL urljump = response.url();


        }catch(Exception e){
            e.printStackTrace();
            logger.error("jsoup获取网站Docment异常", e);
        }
        return doc;
    }

    public static String getDocmentByYm(String ym){
        String  result = null;
        List<String> results = getUrls(ym);
        for(String url : results){
            try {
                Document fp = getWebDocument(url);
                if(fp != null){
                    result = url;
                    break;
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        return result;
    }

//    public static List<Object> getObjects(Document doc, String xpath){
//        List<Object> target = null;
//        if(doc == null)return target;
//        try{
////			System.out.println(doc);
//            JXDocument jxDocument = new JXDocument(doc);
//            List<Object> rs = jxDocument.sel(xpath);
//            target = rs;
//        }catch(Exception e){
//            e.printStackTrace();
//        }
//        return target;
//    }

    public static List<Element> oprateObjects(List<Object> objs){
        List<Element> target = new ArrayList<Element>();
        if(objs != null){
            for(Object obj : objs){
                if(obj != null && obj instanceof Element){
                    target.add((Element)obj);
                }
            }
        }
        return target;
    }

    public static String getDomainFromUrl(String url) {
        url = url.trim();
        if (url.startsWith("https://"))
            url = url.substring(8);
        else if (url.startsWith("http://"))
            url = url.substring(7);
        if (url.startsWith("www."))
            url = url.substring(4);
        int end = url.indexOf("/");
        if (end > 0)
            url = url.substring(0, end);
        end = url.indexOf("?");
        if (end > 0)
            url = url.substring(0, end);
        return url;
    }

    public static String getDomainByUrl(String url){
        String domain = null;
        try{
            String wym = StringParseUtils.parseString(url, "([^\\. ^/ ^\\s]+(\\.)){1,}[a-z]+", false, 0);
            if(wym != null){
                wym = wym.replaceAll("www\\.", "");
                domain = wym;
            }
        }catch(Exception e){
            e.printStackTrace();
        }
        return domain;
    }

    public static String getDomainByUrlWWW(String url){
        String domain = null;
        try{
            String wym = StringParseUtils.parseString(url, "([^\\. ^/ ^\\s]+(\\.)){1,}[a-z]+", false, 0);
            if(wym != null){
                domain = wym;
            }
        }catch(Exception e){
            e.printStackTrace();
        }
        return domain;
    }

    public static String getDomainByUrl(String url, int num){
        String domain = null;
        try{
            String wym = StringParseUtils.parseString(url, "([^\\u4e00-\\u9fa5 ^\\. ^/ ^\\s ^.]+(\\.)){1,}[a-z]+", num);
            if(wym != null){
                wym = wym.replaceAll("www\\.", "");
                domain = wym;
            }
        }catch(Exception e){
            e.printStackTrace();
        }
        return domain;
    }

    public static String getDomainNameNotOpen(String url)
    {
        try {
//			String host = new URL(url).getHost().toLowerCase();//
            //System.out.println(host);
            Pattern pattern = Pattern.compile("(?<=http://|\\.)[^.]*?(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.cc|\\.me|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)");
            Matcher matcher = pattern.matcher(url);
            while (matcher.find()) {
                return matcher.group();
            }

        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return  null;
    }

    public static String getDomainName(String url)
    {
        try {
            String host = new URL(url).getHost().toLowerCase();//
            //System.out.println(host);
            Pattern pattern = Pattern.compile("[^\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.cc|\\.me|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)");
            Matcher matcher = pattern.matcher(host);
            while (matcher.find()) {
                return matcher.group();
            }

        } catch (MalformedURLException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return  null;
    }

    public static String getUrl(String urlStr){
        String url = null;
        try{
            String wym = StringParseUtils.parseString(urlStr, "(http(s)?://)?([^\\u4e00-\\u9fa5 ^\\. ^/ ^\\s ^: ^：]*(\\.)){1,}[a-z]+", false, 0);
            url = wym;
            if(url != null && !url.trim().equalsIgnoreCase("")){
                if(!url.contains("http")){
                    url = "http://" + url;
                }
            }
        }catch(Exception e){
            e.printStackTrace();
        }
        return url;
    }

    public static String getUrl(String urlStr, int num){
        String url = null;
        try{
            if(urlStr == null || urlStr.trim().equalsIgnoreCase(""))return null;
            String wym = StringParseUtils.parseString(urlStr, "(http(s)?://)?([^\\u4e00-\\u9fa5 ^\\. ^/ ^\\s ^\\: ^\\：]*(\\.)){1,}[a-z]+", num);
            url = wym;
            if(url != null){
                if(!url.contains("http")){
                    url = "http://" + url;
                }
            }
        }catch(Exception e){
            e.printStackTrace();
        }
        return url;
    }

    public static void download(String urlString, String filename) throws Exception {
        // 构造URL
        URL url = new URL(urlString);
        // 打开连接
        URLConnection con = url.openConnection();
        // 输入流
        InputStream is = con.getInputStream();
        // 1K的数据缓冲
        byte[] bs = new byte[1024];
        // 读取到的数据长度
        int len;
        // 输出的文件流
        OutputStream os = new FileOutputStream(filename);
        // 开始读取
        while ((len = is.read(bs)) != -1) {
            os.write(bs, 0, len);
        }
        // 完毕，关闭所有链接
        os.close();
        is.close();
    }

    public static String getDomainShortNameByEnd(String domain) {
        String efix = null;
        for (String end : topd) {
            if (domain.endsWith(end)) {
                domain = domain.substring(0, domain.length() - end.length());
                efix = end;
                break;
            }
        }
        if (efix == null) {
            int index = domain.lastIndexOf(".");
            if (index > 0) {
                efix = domain.substring(index);
                domain = domain.substring(0, index);
            } else
                return domain;
        }
        String shortName = domain;
        int index = domain.lastIndexOf(".");
        if (index >= 0) {
            shortName = domain.substring(index + 1);
        }
        return shortName + efix;

    }

    public static HtmlPage getHtmlPageByDomain(String ym){
        HtmlPage page = null;
        List<String> urls = getUrls(ym);
        for(String url : urls){
            page = getHtmlPageByUrl(url);
            if(page != null)break;
        }
        return page;
    }

    public static String getUrlByHtmlPageDomain(String ym){
        String result = null;
        List<String> urls = getUrls(ym);
        for(String url : urls){
            HtmlPage page = getHtmlPageByUrl(url);
            if(page != null){
                result = url;
                break;
            }
        }
        return result;
    }

    public static HtmlPage getHtmlPageByUrl(String url){
        HtmlPage page = null;
        WebClient webClient = null;
        try{
            webClient = new WebClient();
            webClient.getOptions().setJavaScriptEnabled(false);
            webClient.getOptions().setCssEnabled(false);
            webClient.getOptions().setActiveXNative(false);
            webClient.getOptions().setThrowExceptionOnScriptError(false);
            webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
            webClient.getOptions().setUseInsecureSSL(true);
            webClient.getOptions().setTimeout(10 * 1000);
            page = webClient.getPage(url);
        }catch(Exception e){
            e.printStackTrace();
        }finally{
//			if(webClient != null)webClient.close();
        }
        return page;
    }

    public static String sendGetRequest(String reqURL, String decodeCharset){
        long responseLength = 0;       //响应长度
        String responseContent = null; //响应内容
        HttpClient httpClient = new DefaultHttpClient(); //创建默认的httpClient实例
        HttpGet httpGet = new HttpGet(reqURL);           //创建org.apache.http.client.methods.HttpGet
        try{
            HttpResponse response = httpClient.execute(httpGet); //执行GET请求
            HttpEntity entity = response.getEntity();            //获取响应实体
            if(null != entity){
                responseLength = entity.getContentLength();
                responseContent = EntityUtils.toString(entity, decodeCharset==null ? "UTF-8" : decodeCharset);
                EntityUtils.consume(entity); //Consume response content
            }
            System.out.println("请求地址: " + httpGet.getURI());
            System.out.println("响应状态: " + response.getStatusLine());
            System.out.println("响应长度: " + responseLength);
            System.out.println("响应内容: " + responseContent);
        }catch(ClientProtocolException e){
            logger.debug("该异常通常是协议错误导致,比如构造HttpGet对象时传入的协议不对(将'http'写成'htp')或者服务器端返回的内容不符合HTTP协议要求等,堆栈信息如下", e);
        }catch(ParseException e){
            logger.debug(e.getMessage(), e);
        }catch(IOException e){
            logger.debug("该异常通常是网络原因引起的,如HTTP服务器未启动等,堆栈信息如下", e);
        }finally{
            httpClient.getConnectionManager().shutdown(); //关闭连接,释放资源
        }
        return responseContent;
    }

    public static PhantomJSDriver getPhantomJSDriver(){
        PhantomJSDriver target = null;
        DesiredCapabilities dcp = new DesiredCapabilities();
        dcp.setJavascriptEnabled(true);
        dcp.setPlatform(Platform.LINUX);
        dcp.setBrowserName("Chrome");
        dcp.setCapability("Accept-Language", "zh-CN,zh;q=0.8");
        dcp.setCapability("phantomjs.page.settings.resourceTimeout", 5000);
        dcp.setCapability("phantomjs.page.settings.loadImages", false);
        dcp.setCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36");
        dcp.setCapability(PhantomJSDriverService.PHANTOMJS_EXECUTABLE_PATH_PROPERTY, phantomjsPath);
        String [] phantomJsArgs = {"--ignore-ssl-errors=true", "--ssl-protocol=any", "--output-encoding=utf-8", "--script-encoding=utf-8", "--disk-cache=no"};
        dcp.setCapability(PhantomJSDriverService.PHANTOMJS_CLI_ARGS, phantomJsArgs);
        target = new PhantomJSDriver(dcp);
        target.manage().window().setSize(new Dimension(1440, 900));
        target.manage().timeouts().implicitlyWait(3, TimeUnit.SECONDS);
        target.manage().timeouts().pageLoadTimeout(40, TimeUnit.SECONDS);
        target.manage().timeouts().setScriptTimeout(40, TimeUnit.SECONDS);
        return target;
    }

    /**
     * Grom
     * getPhantomJSDriverByPort
     * Dis：赋予端口获取phantomjs进程方法
     * @param port --
     * @return --
     */
    public static PhantomJSDriver getPhantomJSDriverByPort(Integer port){
        String potStr = "--remote-debugger-port="+port;
        PhantomJSDriver target = null;
        DesiredCapabilities dcp = new DesiredCapabilities();
        dcp.setJavascriptEnabled(true);
        dcp.setPlatform(Platform.LINUX);
        dcp.setBrowserName("Chrome");
        dcp.setCapability("Accept-Language", "zh-CN,zh;q=0.8");
        dcp.setCapability("phantomjs.page.settings.resourceTimeout", 5000);
        dcp.setCapability("phantomjs.page.settings.loadImages", false);
        dcp.setCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36");
        dcp.setCapability(PhantomJSDriverService.PHANTOMJS_EXECUTABLE_PATH_PROPERTY, phantomjsPath);
        String [] phantomJsArgs = {"--ignore-ssl-errors=true", "--ssl-protocol=any", "--output-encoding=utf-8", "--script-encoding=utf-8", "--disk-cache=no", potStr};
        dcp.setCapability(PhantomJSDriverService.PHANTOMJS_CLI_ARGS, phantomJsArgs);
        target = new PhantomJSDriver(dcp);
        target.manage().window().setSize(new Dimension(1440, 900));
        target.manage().timeouts().implicitlyWait(3, TimeUnit.SECONDS);
        target.manage().timeouts().pageLoadTimeout(40, TimeUnit.SECONDS);
        target.manage().timeouts().setScriptTimeout(40, TimeUnit.SECONDS);
        return target;
    }

    /**
     *
     * @param plat
     * @param browserName
     * @param javasriptenabled
     * @param phantomjsPath
     * @param capabilitys
     * @param phantomJsArgs
     * @param dimensionW
     * @param dimensionH
     * @return
     */
    public static PhantomJSDriver getPhantomJSDriver(Platform plat, String browserName, boolean javasriptenabled, String phantomjsPath, Map<String, Object> capabilitys, String[] phantomJsArgs, Integer dimensionW, Integer dimensionH){
        PhantomJSDriver target = null;
        DesiredCapabilities dcp = new DesiredCapabilities();
        dcp.setJavascriptEnabled(javasriptenabled);
        dcp.setPlatform(plat);
        dcp.setBrowserName(browserName);
        if(capabilitys != null && !capabilitys.isEmpty()){
            for(String key : capabilitys.keySet()){
                dcp.setCapability(key, capabilitys.get(key));
            }
        }
        if(dimensionW == null || dimensionH == null || dimensionW == 0 || dimensionH == 0){
            dimensionW = 1440;
            dimensionH = 900;
        }
        dcp.setCapability(PhantomJSDriverService.PHANTOMJS_CLI_ARGS, phantomJsArgs);
        target = new PhantomJSDriver(dcp);
        target.manage().window().setSize(new Dimension(dimensionW, dimensionH));
        return target;
    }

    public static String getPhantomJSHtmlByUrl(String url){
        String html = null;
        PhantomJSDriver driver = null;
        try{
            driver = getPhantomJSDriver();
            driver.get(url);
            html = driver.getPageSource();
            Thread.sleep(5000);
            WebElement ele = driver.findElementByCssSelector("body > div:nth-child(3) > div > div > div.left_part > div > div:nth-child(1) > h2 > a");
            if(ele != null){
                System.out.println("发现该元素");
            }
        }catch(Exception e){
            e.printStackTrace();
        }finally{
            if(driver != null){
                driver.close();
                driver.quit();
            }
        }
        return html;
    }

    /**
     * CSS单网页定向采集
     * @param url
     * @param detail
     * @param openSleep
     * @return
     */
    public static Map<String, String> directionalCollectByCss(String url, Map<String, String> detail, long openSleep){
        Map<String, String> result = new HashMap<String, String>();

        return result;
    }

    public static void pressCtrlF(PhantomJSDriver driver){
        try{
            System.out.println(Keys.LEFT_CONTROL);
            driver.getKeyboard().pressKey(Keys.LEFT_CONTROL);
            Thread.sleep(100);
            driver.getKeyboard().pressKey("f");
            Thread.sleep(100);
            driver.getKeyboard().releaseKey("f");
            Thread.sleep(100);
            driver.getKeyboard().releaseKey(Keys.LEFT_CONTROL);
        }catch(Exception e){
            e.printStackTrace();
        }
    }

    public static void pressEnter(PhantomJSDriver driver){
        try{
            System.out.println(Keys.ENTER);
            driver.getKeyboard().pressKey(Keys.ENTER);
            Thread.sleep(100);
            driver.getKeyboard().releaseKey(Keys.ENTER);
        }catch(Exception e){
            e.printStackTrace();
        }
    }

    public static boolean writePictureByWebUrl(String url, String destFileDir, String fileName, String keyword){
        boolean target = false;
        PhantomJSDriver driver = null;
        try{
            driver = getPhantomJSDriver();
            driver.navigate().to(url);
            Thread.sleep(10000);
            if(keyword != null && !keyword.trim().equalsIgnoreCase("")){
                driver.executeScript("window.find('" + keyword + "',false,false);", "");
            }
            byte[] images = driver.getScreenshotAs(OutputType.BYTES);
            File destfile = new File(destFileDir);
            if(!destfile.exists())destfile.mkdirs();
            File rdestFile = new File(destFileDir + "/" + fileName);
            if(rdestFile.exists())rdestFile.delete();
            FileUtils.writeByteArrayToFile(rdestFile, images);
            if(rdestFile.exists())target = true;
        }catch(Exception e){
            logger.error("获取网页图片异常", e);
        }finally{
            if(driver != null){
                try{
                    driver.close();
                }catch(Exception e){
                    e.printStackTrace();
                }
                try{
                    driver.quit();
                }catch(Exception e){
                    e.printStackTrace();
                }
            }
        }
        return target;
    }


    /**
     * fileName 文件名(不带后缀)
     * filePath 文件的本地路径 (xxx / xx / test.jpg)
     */
    public static void uploadFile(String fileName, String filePath, String host) {

        HttpURLConnection conn = null;

        /// boundary就是request头和上传文件内容的分隔符(可自定义任意一组字符串)
        String BOUNDARY = "******";
        // 用来标识payLoad+文件流的起始位置和终止位置(相当于一个协议,告诉你从哪开始,从哪结束)
        String  preFix = ("\r\n--" + BOUNDARY + "--\r\n");

        try {
            // (HttpConst.uploadImage 上传到服务器的地址
            URL url = new URL(host);
            conn = (HttpURLConnection) url.openConnection();
            conn.setConnectTimeout(5000);
            conn.setReadTimeout(30000);
            conn.setDoOutput(true);
            conn.setDoInput(true);
            conn.setUseCaches(false);
            // 设置请求方法
            conn.setRequestMethod("POST");
            // 设置header
            conn.setRequestProperty("Accept","*/*");
            conn.setRequestProperty("Connection", "keep-alive");
            conn.setRequestProperty("Content-Type",
                    "multipart/form-data; boundary=" + BOUNDARY);
            // 获取写输入流
            OutputStream out = new DataOutputStream(conn.getOutputStream());
            // 获取上传文件
            File file = new File(filePath);

            // 要上传的数据
            StringBuffer strBuf = new StringBuffer();

            // 标识payLoad + 文件流的起始位置
            strBuf.append(preFix);

            // 下面这三行代码,用来标识服务器表单接收文件的name和filename的格式
            // 在这里,我们是file和filename.后缀[后缀是必须的]。
            // 这里的fileName必须加个.jpg,因为后台会判断这个东西。
            // 这里的Content-Type的类型,必须与fileName的后缀一致。
            // 如果不太明白,可以问一下后台同事,反正这里的name和fileName得与后台协定！
            // 这里只要把.jpg改成.txt，把Content-Type改成上传文本的类型，就能上传txt文件了。
            strBuf.append("Content-Disposition: form-data; name=\"file\"; filename=\"" + fileName+".jpg" + "\"\r\n");
            strBuf.append("Content-Type: image/jpeg"  + "\r\n\r\n");
            out.write(strBuf.toString().getBytes());

            // 获取文件流
            FileInputStream fileInputStream = new FileInputStream(file);
            DataInputStream inputStream = new DataInputStream(fileInputStream);

            // 每次上传文件的大小(文件会被拆成几份上传)
            int bytes = 0;
            // 计算上传进度
            float count = 0;
            // 获取文件总大小
            int fileSize = fileInputStream.available();
            // 每次上传的大小
            byte[] bufferOut = new byte[1024];
            // 上传文件
            while ((bytes = inputStream.read(bufferOut)) != -1) {
                // 上传文件(一份)
                out.write(bufferOut, 0, bytes);
                // 计算当前已上传的大小
                count += bytes;
                // 打印上传文件进度(已上传除以总大*100就是进度)
                logger.info("progress:" +(count / fileSize * 100) +"%");
            }

            // 关闭文件流
            inputStream.close();

            // 标识payLoad + 文件流的结尾位置
            out.write(preFix.getBytes());

            // 至此上传代码完毕

            // 总结上传数据的流程：preFix + payLoad(标识服务器表单接收文件的格式) + 文件(以流的形式) + preFix
            // 文本与图片的不同,仅仅只在payLoad那一处的后缀的不同而已。

            // 输出所有数据到服务器
            out.flush();

            // 关闭网络输出流
            out.close();

            // 重新构造一个StringBuffer,用来存放从服务器获取到的数据
            strBuf = new StringBuffer();

            // 打开输入流 , 读取服务器返回的数据
            BufferedReader reader = new BufferedReader(new
                    InputStreamReader(conn.getInputStream()));

            String line;

            // 一行一行的读取服务器返回的数据
            while ((line = reader.readLine()) != null) {
                strBuf.append(line).append("\n");
            }

            // 关闭输入流
            reader.close();

            // 打印服务器返回的数据
            logger.info("上传成功:"+strBuf.toString());

        } catch (Exception e) {
            logger.info("上传图片出错:"+e.toString());
        } finally {
            if (conn != null) {
                conn.disconnect();
            }
        }

    }

//    public static WebInfo phantomJsOpenHtml(String url){
//        WebInfo result = null;
//        String html = null;
//        WebCrawlUtils utils = new WebCrawlUtils();
//        try{
//            boolean init_status = utils.initDriverByPoolsDefault();
//            if(!init_status)return result;
////			utils.openWebPage(url);
////			utils.ajaxWait(5000);
//            PhantomJsWebInfo webinfo = SiteInLinkUtil.getPhantomJsWebInfo(utils, url, false, new ArrayList<String>(), false);
//            if(webinfo == null)return result;
//            html = webinfo.getHtml();
//            result = new WebInfo();
//            result.setUrl(url);
//            result.setCurrent_url(url);
//            result.setHtml(html);
//        }catch(Exception e){
//            logger.error("phantomJs打开网站:{}异常", url, e);
//        }finally{
//            if(utils != null){
//                utils.recoveryDriver();
//            }
//        }
//        return result;
//    }

//    public static WebInfo phantomJsOpenHtmlTest(String url){
//        WebInfo result = null;
//        String html = null;
//        WebCrawlUtils utils = new WebCrawlUtils();
//        try{
//            utils.initDriverDefault();
////			utils.openWebPage(url);
////			utils.ajaxWait(5000);
//            PhantomJsWebInfo webinfo = SiteInLinkUtil.getPhantomJsWebInfo(utils, url, false, new ArrayList<String>(), false);
//            if(webinfo == null) return result;
//            html = webinfo.getHtml();
//            result = new WebInfo();
//            result.setUrl(url);
//            result.setCurrent_url(url);
//            result.setHtml(html);
//        }catch(Exception e){
//            logger.error("phantomJs打开网站:{}异常", url, e);
//        }finally{
//            if(utils != null){
//                utils.close();
//                utils.quit();
//            }
//        }
//        return result;
//    }

    public static WebInfo opreateSite(String ym, int open_num, int timeout){
        WebInfo info = null;
        List<String> urls = HttpUtil.getUrls(ym);
        for(String rurl : urls){
            info = opreateSite(ym, rurl, open_num, timeout);
            if(info != null)break;
        }
        return info;
    }

    public static WebInfo opreateSite(String ym, String url, int open_num, int timeout){
        WebInfo info = null;
        for(int i = 0 ; i < open_num ; i ++){
            info = openSite(ym, url, (i + 1) * timeout);
            if(info != null)break;
        }
        return info;
    }

    public static WebInfo openSite(String ym, String rurl, int timeout){
        WebInfo result = null;
        try {
            Response resp = HttpUtil.getResponse(rurl, timeout);
            Document doc = resp.parse();
            String html = doc.html();
            boolean jump = HttpUtil.webJump(doc);
            String current_url = null;
            current_url = rurl;
//			current_url = resp.url().toString();
            if(html != null && current_url != null){
                result = new WebInfo();
                result.setCurrent_url(current_url);
                result.setUrl(rurl);
                result.setHtml(html);
                result.setJump(jump);
                result.setResp(resp);
            }
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
            logger.error("打开网站：{}异常", rurl);
        }
        return result;
    }

    public static String getChecksum(String filename, String algorithm) throws Exception {
        byte[] b = createChecksum(filename, algorithm);
        String result = "";
        for (int i=0; i < b.length; i++) {
            result += Integer.toString( ( b[i] & 0xff ) + 0x100, 16).substring( 1 );
        }
        return result;
    }

    public static String getChecksum(byte[] buffer, String algorithm) throws Exception {
        byte[] b = createChecksum(buffer, algorithm);
        String result = "";
        for (int i=0; i < b.length; i++) {
            result += Integer.toString( ( b[i] & 0xff ) + 0x100, 16).substring( 1 );
        }
        return result;
    }

    public static byte[] createChecksum(String filename, String algorithm) throws Exception {
        InputStream fis =  new FileInputStream(filename);
        byte[] buffer = new byte[1024];
        MessageDigest complete = MessageDigest.getInstance(algorithm);
        int numRead;
        do {
            numRead = fis.read(buffer);
            if (numRead > 0) {
                complete.update(buffer, 0, numRead);
            }
        } while (numRead != -1);
        fis.close();
        return complete.digest();
    }

    public static byte[] createChecksum(byte[] buffer, String algorithm) throws Exception {
        MessageDigest complete = MessageDigest.getInstance(algorithm);
        complete.update(buffer, 0, buffer.length);
        return complete.digest();
    }

    public static String clawerSyncJsoupPost(String op_type, String taskId){
        String result = null;
        try{
            Connection con = Jsoup.connect(crawlerSyncUrl);
            con.validateTLSCertificates(false);
            con.timeout(30000);
            con.ignoreContentType(true);
            con.data("op_type", op_type);
            con.data("task_id", taskId);
            Document doc = con.post();
            result = doc.body().text();
        }catch(Exception e){
            logger.error("同步任务数据异常：{}", taskId, e);
        }
        return result;
    }


    public static void downloadPictureByUrl(String urlStr, String destDir, String fileName, String format){
        try {
            URL url = new URL(urlStr);
            DataInputStream dataInputStream = new DataInputStream(url.openStream());
            String imageName = destDir + fileName + format;
            FileOutputStream fileOutputStream = new FileOutputStream(new File(imageName));

            byte[] buffer = new byte[1024];
            int length;

            while ((length = dataInputStream.read(buffer)) > 0) {
                fileOutputStream.write(buffer, 0, length);
            }

            dataInputStream.close();
            fileOutputStream.close();
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static void downloadMIAPPLogo(String appfxdz, String destDir, String fileName, String format){
        Document doc = HttpUtil.getWebDocument(appfxdz);
        Elements eles = doc.getElementsByClass("yellow-flower");
        if(eles.size() > 0){
            String urlStr = eles.get(0).attr("src");
            System.out.println(urlStr);
            downloadPictureByUrl(urlStr, destDir, fileName, format);
        }
    }

    public static void downloadBAIDUAPPLogo(String appfxdz, String destDir, String fileName, String format){
        Document doc = HttpUtil.getWebDocument(appfxdz);
        Elements eles = doc.getElementsByClass("app-pic");
        if(eles.size() > 0){
            String url = eles.get(0).getElementsByIndexEquals(0).attr("src");
            System.out.println(url);
            downloadPictureByUrl(url, destDir, fileName, format);
        }
    }

    public static void download360APPLogo(String appfxdz, String destDir, String fileName, String format){
        Document doc = HttpUtil.getWebDocument(appfxdz);
        Elements eles = doc.getElementsByClass("clearfix");
        if(eles.size() > 0){
            String url = eles.get(0).getElementsByIndexEquals(0).attr("src");
            System.out.println(url);
            downloadPictureByUrl(url, destDir, fileName, format);
        }
    }

    public static void downloadQQAPPLogo(String appfxdz, String destDir, String fileName, String format){
        Document doc = HttpUtil.getWebDocument(appfxdz);
        Elements eles = doc.getElementsByClass("det-icon");
        if(eles.size() > 0){
            String url = eles.get(0).getElementsByIndexEquals(0).attr("src");
            System.out.println(url);
            downloadPictureByUrl(url, destDir, fileName, format);
        }
    }

}
